{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.Change mode  to test mode \n",
    "```\n",
    "commands add \n",
    "--phase-test True\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.Verify the performance of the model in the test dataset \n",
    "Run the following command, you can see the performance of the model in the test set in the ./submission/ directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/fishnet150-32.yaml\" --resume ./checkpoints/fishnet150_bs32/_15.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/fishnet150-32.yaml\" --resume ./checkpoints/fishnet150_bs32/_51_best.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/fishnet150-32.yaml\" --resume ./checkpoints/fishnet150_bs32/_26_best.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/FeatherNet54-32.yaml\" --resume ./checkpoints/FeatherNet54/_40_best.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/FeatherNet54-se-64.yaml\" --resume ./checkpoints/FeatherNet54-se/_68_best.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/mobilenetv2.yaml\" --resume ./checkpoints/mobilenetv2_bs32/_4_best.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/mobilenetv2.yaml\" --resume ./checkpoints/mobilenetv2_bs32/_5.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/mobilenetv2.yaml\" --resume ./checkpoints/mobilenetv2_bs32/_6.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/MobileLiteNetA-32.yaml\" --resume ./checkpoints/mobilelitenetA_bs32/_50_best.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/MobileLiteNetB-32.yaml\" --resume ./checkpoints/mobilelitenetB_bs32/_47_best.pth.tar --phase-test True --val True --val-save True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.split the predicted scores from each submission file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def splitscore(file_dir):\n",
    "    score = []\n",
    "    Prefix_str = []\n",
    "    f = open(file_dir)\n",
    "    for line in f:\n",
    "        s =line.split()\n",
    "        score.append(float(s[-1]))\n",
    "        s = s[0] + ' ' + s[1] + ' ' + s[2] + ' '\n",
    "        Prefix_str.append(s)\n",
    "    return score,Prefix_str"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.Collect all predicted scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_dir1='submission/2019-01-28_15:45:05_fishnet150_52_submission.txt'\n",
    "score1,Prefix_str = splitscore(file_dir1)\n",
    "file_dir2 = 'submission/2019-02-13_15:22:05_FeatherNet54-se_69_submission.txt'\n",
    "score2,Prefix_str = splitscore(file_dir2)\n",
    "# print(Prefix_str[1])\n",
    "file_dir3 = 'submission/2019-03-01_22:25:43_fishnet150_27_submission.txt'\n",
    "score3,Prefix_str = splitscore(file_dir3)\n",
    "# \n",
    "file_dir4 = 'submission/2019-02-13_13:30:12_FeatherNet54_41_submission.txt'\n",
    "score4,Prefix_str = splitscore(file_dir4)\n",
    "# \n",
    "file_dir5 = 'submission/2019-02-13_14:13:43_fishnet150_16_submission.txt'\n",
    "score5,Prefix_str = splitscore(file_dir5)\n",
    "\n",
    "file_dir6 = 'submission/2019-02-16_19:31:04_moilenetv2_5_submission.txt'\n",
    "score6,Prefix_str = splitscore(file_dir6)\n",
    "file_dir7 = 'submission/2019-02-16_19:30:02_moilenetv2_7_submission.txt'\n",
    "score7,Prefix_str = splitscore(file_dir7)\n",
    "file_dir8 = 'submission/2019-02-16_19:28:47_moilenetv2_6_submission.txt'\n",
    "score8,Prefix_str = splitscore(file_dir8)\n",
    "\n",
    "\n",
    "file_dir9 = 'submission/2019-03-01_17:10:11_mobilelitenetB_48_submission.txt'\n",
    "score9,Prefix_str = splitscore(file_dir9)\n",
    "file_dir10 = 'submission/2019-03-01_17:38:27_mobilelitenetA_51_submission.txt'\n",
    "score10,Prefix_str = splitscore(file_dir10)\n",
    "\n",
    "# scores =[score1,score2,score3,score4,score5,score6,score7,score8,score9]\n",
    "scores = [score1,score2,score3,score4,score5,score6,score7,score8,score9,score10]\n",
    "print(scores[0][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.if you have private test dataset label list run 5-1 else run 5-2 directly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def Average(lst): \n",
    "    return sum(lst) / len(lst) \n",
    "def fecth_ensembled_score(scores, threshold):\n",
    "    ensembled_score  = []\n",
    "    for i in range(len(score1)):\n",
    "        line_socres = [scores[j][i] for j in range(len(scores))]\n",
    "        mean_socre = Average(line_socres)\n",
    "        if mean_socre > threshold:\n",
    "            ensembled_score.append(max(line_socres))\n",
    "        else:\n",
    "            ensembled_score.append(min(line_socres))\n",
    "    return ensembled_score\n",
    "## \n",
    "def num_err(ensembled_score,threshold,real_scores):            \n",
    "    count = 0\n",
    "    for i in range(len(real_scores)):\n",
    "        if real_scores[i] == (ensembled_score[i]>0.5):\n",
    "            pass\n",
    "        else:\n",
    "            count = count + 1\n",
    "    if count < 30:\n",
    "        print('threshold: {:.3f} num_errors is {}'.format(threshold,count))\n",
    "    return count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-1.Get test labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_ensembled_file_dir='data/test_private_list.txt'\n",
    "# submission_ensembled_file_dir='data/val_label.txt'\n",
    "real_scores,Prefix_str = splitscore(submission_ensembled_file_dir)\n",
    "len(real_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_threshold():\n",
    "    min_count = 10000000\n",
    "    best_threshold = 0.0\n",
    "    for i in range(100):\n",
    "        threshold = i / 100\n",
    "        ensembled_score = fecth_ensembled_score(scores, threshold)\n",
    "        count = num_err(ensembled_score,threshold,real_scores)\n",
    "        if count < min_count:\n",
    "            min_count = count\n",
    "            best_threshold = threshold\n",
    "    return best_threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-2.Select the appropriate threshold to generate the final submission\n",
    "**ntice**: example threshold=0.53"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_threshold = get_best_threshold()\n",
    "print('best threshold is :',best_threshold)\n",
    "submission_ensembled_file_dir='submission/final_submission.txt'\n",
    "ensembled_file = open(submission_ensembled_file_dir,'a')\n",
    "ensembled_score = fecth_ensembled_score(scores, best_threshold)\n",
    "for i in range(len(ensembled_score)):\n",
    "    ensembled_file.write(Prefix_str[i]+str(ensembled_score[i])+'\\n')\n",
    "ensembled_file.close()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
